import time

from lxml import etree

import scrapy
from selenium import webdriver

# 网安
class ScuccsSpider(scrapy.Spider):
    name = 'scuccs_spider'
    allowed_domains = ['ccs.scu.edu.cn']
    start_urls = ['https://ccs.scu.edu.cn/']

    def parse(self, response):
        # 通知
        yield scrapy.Request('https://ccs.scu.edu.cn/zxdt/tzgg.htm', callback=self.parse_tz)
        # 新闻
        yield scrapy.Request('https://ccs.scu.edu.cn/zxdt/xwdt.htm', callback=self.parse_jwcxw)

    # 通知
    def parse_tz(self,response):
        # 文章url
        urls = response.xpath('//div[@class="zxdt-list"]//li//a/@href').extract()
        for url in urls:
            if url[0:3] == '../':
                url = url[3:]
                if url[0:3] == '../':
                    url = url[3:]
                print(url)
            yield scrapy.Request(self.start_urls[0]+url, callback=self.parse_article)
        # 下页
        nextUrl = response.xpath('//span[@class="p_next p_fun"]/a/@href').extract_first()
        if nextUrl == None:
            pass
        else:
            if nextUrl[0:5] == 'tzgg/':
                yield scrapy.Request("https://ccs.scu.edu.cn/zxdt/" + nextUrl, callback=self.parse_tz)
            else:
                yield scrapy.Request("https://ccs.scu.edu.cn/zxdt/" + 'tzgg/' + nextUrl, callback=self.parse_tz)


    # 新闻
    def parse_xw(self,response):
        # 文章url
        urls = response.xpath('//div[@class="zxdt-list"]//li//a/@href').extract()
        for url in urls:
            if url[0:3] == '../':
                url = url[3:]
                if url[0:3] == '../':
                    url = url[3:]
                print(url)
            yield scrapy.Request(self.start_urls[0]+url, callback=self.parse_article)
        # 下页
        nextUrl = response.xpath('//span[@class="p_next p_fun"]/a/@href').extract_first()
        if nextUrl == None:
            pass
        else:
            if nextUrl[0:5] == 'xwdt/':
                yield scrapy.Request("https://ccs.scu.edu.cn/zxdt/" + nextUrl, callback=self.parse_xw)
            else:
                yield scrapy.Request("https://ccs.scu.edu.cn/zxdt/" + 'xwdt/' + nextUrl, callback=self.parse_xw)

    def parse_article(self,response):
        title = response.xpath('//div[@class="xwdt"]//text()').extract_first()
        date = response.xpath('//p[@class="tit-p"]/text()').extract_first()[3:13]
        contentArray = response.xpath('//div[@class="v_news_content"]//p//text()').extract()
        # if len(contentArray) == 0:
        #     contentArray = response.xpath('//div[@class="page-content"]//text()').extract()
        content = "".join(contentArray)
        pic = response.xpath('//div[@class="v_news_content"]//img/@src').extract_first()
        # if pic == None:
        #     pic = response.xpath('//div[@class ="page-content"]//img/@src').extract_first()
        if pic == None:
            pic = ""
        else:
            pic = "https://ccs.scu.edu.cn/" + pic

        yield{
            'title':title,
            'date':date,
            'content':content,
            'pic':pic,
            'url':response.url,
            'source':'网络空间安全学院'
        }








